#----------------------------------*-sh-*--------------------------------------
# =========                 |
# \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
#  \\    /   O peration     |
#   \\  /    A nd           | www.openfoam.com
#    \\/     M anipulation  |
#------------------------------------------------------------------------------
#     Copyright (C) 2011-2016 OpenFOAM Foundation
#     Copyright (C) 2017-2020 OpenCFD Ltd.
#------------------------------------------------------------------------------
# License
#     This file is part of OpenFOAM, distributed under GPL-3.0-or-later.
#
# File
#     etc/config.csh/mpi
#     - sourced by OpenFOAM-*/etc/cshrc
#
# Description
#     Setup for MPI communications library for OpenFOAM
#
#     User adjustments are possible in these files:
#       - config.csh/openmpi-system
#       - config.csh/openmpi
#       - config.csh/mpi-user
#       - config.csh/mpi-system
#
#     For USERMPI, the user is responsible for supplying an appropriate
#     wmake/rules/General/mplibUSERMPI file and managing all settings
#
#     For INTELMPI, note that the (I_MPI_CC, I_MPI_CCX) environment variables
#     define the underlying compiler to be used
#
#------------------------------------------------------------------------------
unsetenv MPI_ARCH_PATH MPI_HOME
setenv FOAM_MPI dummy  # Fallback value

switch ("$WM_MPLIB")
case SYSTEMOPENMPI*:
    # The system installed openmpi, locations discovery via mpicc.
    setenv FOAM_MPI openmpi-system
    _foamEtc -config openmpi-system             # <- Adjustments (optional)

    # Respect MPI_ARCH_PATH if set to valid directory (eg, user adjustments)
    if (! $?MPI_ARCH_PATH ) setenv MPI_ARCH_PATH
    if ( -d "$MPI_ARCH_PATH" ) then
        _foamAddLibAuto $MPI_ARCH_PATH
    else
        # Slight hack: strip off 'lib' to get presumed prefix for include and libs
        set libDir=`mpicc --showme:link | sed -e 's/.*-L\([^ ]*\).*/\1/'`

        setenv MPI_ARCH_PATH "${libDir:h}"
        _foamAddLib $libDir
        unset libDir
    endif
    breaksw

case OPENMPI:
    setenv FOAM_MPI openmpi-4.0.3
    _foamEtc -config openmpi                    # <- Adjustments (optional)

    # Respect MPI_ARCH_PATH if set to valid directory (eg, user adjustments)
    if (! $?MPI_ARCH_PATH ) setenv MPI_ARCH_PATH
    if (! -d "$MPI_ARCH_PATH" ) then
        setenv MPI_ARCH_PATH $WM_THIRD_PARTY_DIR/platforms/$WM_ARCH$WM_COMPILER/$FOAM_MPI
    endif

    # Inform openmpi where to find its install directory
    setenv OPAL_PREFIX $MPI_ARCH_PATH

    if ($?FOAM_VERBOSE && $?prompt) then
        echo "Using $WM_MPLIB"
        echo "    FOAM_MPI    : $FOAM_MPI"
        echo "    OPAL_PREFIX : $MPI_ARCH_PATH"
    endif

    _foamAddPath    $MPI_ARCH_PATH/bin
    _foamAddMan     $MPI_ARCH_PATH/share/man
    _foamAddLibAuto $MPI_ARCH_PATH  lib$WM_COMPILER_LIB_ARCH
    breaksw

case USERMPI:
    # Arbitrary, user-specified mpi implementation
    setenv FOAM_MPI mpi-user
    _foamEtc -config mpi-user                   # <- Adjustments (optional)
    breaksw

case SYSTEMMPI:
    setenv FOAM_MPI mpi-system
    _foamEtc -config mpi-system                 # <- Adjustments (optional)

    if ( ! $?MPI_ROOT ) then
        echo
        echo "Warning in $WM_PROJECT_DIR/etc/config.csh/mpi:"
        echo "    Please set the environment variable MPI_ROOT to point to" \
             " the base folder for the system MPI in use."
        echo "    Example:"
        echo
        echo "        setenv MPI_ROOT /opt/mpi"
        echo
    else
        setenv MPI_ARCH_PATH $MPI_ROOT

        if ( ! $?MPI_ARCH_FLAGS ) then
            echo
            echo "Warning in $WM_PROJECT_DIR/etc/config.csh/mpi:"
            echo "    MPI_ARCH_FLAGS is not set. Example:"
            echo
            echo '        setenv MPI_ARCH_FLAGS "-DOMPI_SKIP_MPICXX"'
            echo
        endif

        if ( ! $?MPI_ARCH_INC ) then
            echo
            echo "Warning in $WM_PROJECT_DIR/etc/config.csh/mpi:"
            echo "    MPI_ARCH_INC is not set. Example:"
            echo
            echo '        setenv MPI_ARCH_INC "-isystem $MPI_ROOT/include"'
            echo
        endif

        if ( ! $?MPI_ARCH_LIBS ) then
            echo
            echo "Warning in $WM_PROJECT_DIR/etc/config.csh/mpi:"
            echo "    MPI_ARCH_LIBS is not set. Example:"
            echo
            echo '        setenv MPI_ARCH_LIBS "-L$MPI_ROOT/lib -lmpi"'
            echo
        endif
    endif
    breaksw

case MPICH:
    setenv FOAM_MPI mpich-3.3
    setenv MPI_ARCH_PATH $WM_THIRD_PARTY_DIR/platforms/$WM_ARCH$WM_COMPILER/$FOAM_MPI
    setenv MPI_HOME $MPI_ARCH_PATH

    _foamAddPath    $MPI_ARCH_PATH/bin
    _foamAddMan     $MPI_ARCH_PATH/share/man
    _foamAddLibAuto $MPI_ARCH_PATH  lib$WM_COMPILER_LIB_ARCH
    breaksw

case MPICH-GM:
    setenv FOAM_MPI mpich-gm
    setenv MPI_ARCH_PATH /opt/mpi
    setenv MPICH_PATH $MPI_ARCH_PATH
    setenv GM_LIB_PATH /opt/gm/lib$WM_COMPILER_LIB_ARCH

    _foamAddPath    $MPI_ARCH_PATH/bin
    _foamAddLib     $MPI_ARCH_PATH/lib$WM_COMPILER_LIB_ARCH
    _foamAddLib     $GM_LIB_PATH
    breaksw

case MVA2MPI:  # MVAMPICH2
    setenv FOAM_MPI mvapich2-2.3

    _foamAddPath    $MPI_ARCH_PATH/bin
    _foamAddMan     $MPI_ARCH_PATH/share/man
    _foamAddLibAuto $MPI_ARCH_PATH  lib$WM_COMPILER_LIB_ARCH
    breaksw

case CRAY-MPICH:
    if ( ! $?MPICH_DIR ) setenv MPICH_DIR /dev/null
    setenv FOAM_MPI cray-mpich
    setenv MPI_ARCH_PATH $MPICH_DIR

    if ($?FOAM_VERBOSE && $?prompt) then
        echo "Using $WM_MPLIB"
        echo "    FOAM_MPI  : $FOAM_MPI"
        echo "    MPICH_DIR : $MPI_ARCH_PATH"
    endif

    if ( ! -d "$MPI_ARCH_PATH" ) then
        echo "Warning in $WM_PROJECT_DIR/etc/config.csh/mpi:"
        echo "    Not a valid $WM_MPLIB installation directory."
        echo "    Please set MPICH_DIR properly."
        echo "    Currently using '$MPI_ARCH_PATH'"
    endif

    # _foamAddPath    $MPI_ARCH_PATH/bin
    _foamAddLib     $MPI_ARCH_PATH/lib
    breaksw

case HPMPI:
    setenv FOAM_MPI hpmpi
    setenv MPI_HOME /opt/hpmpi
    setenv MPI_ARCH_PATH $MPI_HOME

    _foamAddPath $MPI_ARCH_PATH/bin

    switch (`uname -m`)
    case i686:
        _foamAddLib $MPI_ARCH_PATH/lib/linux_ia32
        breaksw
    case x86_64:
        _foamAddLib $MPI_ARCH_PATH/lib/linux_amd64
        breaksw
    case ia64:
        _foamAddLib $MPI_ARCH_PATH/lib/linux_ia64
        breaksw
    default:
        echo Unknown processor type `uname -m` for Linux
        breaksw
    endsw
    breaksw

case MPI:
    setenv FOAM_MPI mpi
    setenv MPI_ARCH_PATH /opt/mpi
    breaksw

case FJMPI:
    setenv FOAM_MPI fjmpi
    setenv MPI_ARCH_PATH /opt/FJSVmpi2

    _foamAddPath    $MPI_ARCH_PATH/bin
    _foamAddLib     $MPI_ARCH_PATH/lib/sparcv9
    _foamAddLib     /opt/FSUNf90/lib/sparcv9
    _foamAddLib     /opt/FJSVpnidt/lib
    breaksw

case QSMPI:
    setenv FOAM_MPI qsmpi
    setenv MPI_ARCH_PATH /usr/lib/mpi

    _foamAddPath    $MPI_ARCH_PATH/bin
    _foamAddLib     $MPI_ARCH_PATH/lib
    breaksw

case SGIMPI:
    if ( ! $?MPI_ROOT ) setenv MPI_ROOT /dummy

    if ( ! -d "$MPI_ROOT" ) then
        echo "Warning in $WM_PROJECT_DIR/etc/config.csh/mpi:"
        echo "    Not a valid $WM_MPLIB installation directory."
        echo "    Please set MPI_ROOT properly (usually via the mpt module)"
        echo "    Currently using '$MPI_ARCH_PATH'"
    endif

    if ( "${MPI_ROOT:h}/" == $MPI_ROOT ) then
        setenv MPI_ROOT ${MPI_ROOT:h}
    endif

    setenv FOAM_MPI ${MPI_ROOT:t}
    setenv MPI_ARCH_PATH $MPI_ROOT

    if ($?FOAM_VERBOSE && $?prompt) then
        echo "Using $WM_MPLIB"
        echo "    FOAM_MPI : $FOAM_MPI"
        echo "    MPI_ROOT : $MPI_ARCH_PATH"
    endif

    _foamAddPath    $MPI_ARCH_PATH/bin
    _foamAddLib     $MPI_ARCH_PATH/lib
    breaksw

case INTELMPI*:
    if ( $?I_MPI_ROOT ) then
        # I_MPI_ROOT: The Intel MPI Library installation directory

        # Remove trailing slash
        if ( "${I_MPI_ROOT:h}/" == $I_MPI_ROOT ) then
            setenv I_MPI_ROOT ${I_MPI_ROOT:h}
        endif
        setenv MPI_ARCH_PATH $I_MPI_ROOT
        setenv FOAM_MPI ${MPI_ARCH_PATH:t}

        # If subdirectory is version number only, prefix with 'impi-'
        switch ("$FOAM_MPI")
        case [0-9]*:
            setenv FOAM_MPI "impi-$FOAM_MPI"
            breaksw
        endsw

        if ($?FOAM_VERBOSE && $?prompt) then
            echo "Using $WM_MPLIB"
            echo "    FOAM_MPI   : $FOAM_MPI"
            echo "    I_MPI_ROOT : $MPI_ARCH_PATH"
        endif
    else if ( $?MPI_ROOT ) then
        # MPI_ROOT: General specification

        # Remove trailing slash
        if ( "${MPI_ROOT:h}/" == $MPI_ROOT ) then
            setenv MPI_ROOT ${MPI_ROOT:h}
        endif

        setenv MPI_ARCH_PATH $MPI_ROOT
        setenv FOAM_MPI ${MPI_ARCH_PATH:t}

        # If subdirectory is version number only, prefix with 'impi-'
        switch ("$FOAM_MPI")
        case [0-9]*:
            setenv FOAM_MPI "impi-$FOAM_MPI"
            breaksw
        endsw

        if ($?FOAM_VERBOSE && $?prompt) then
            echo "Using $WM_MPLIB"
            echo "    FOAM_MPI : $FOAM_MPI"
            echo "    MPI_ROOT : $MPI_ARCH_PATH"
        endif
    else
        setenv MPI_ARCH_PATH /dummy
    endif

    if ( ! -d "$MPI_ARCH_PATH" ) then
        echo "Warning in $WM_PROJECT_DIR/etc/config.csh/mpi:"
        echo "    Not a valid $WM_MPLIB installation directory."
        echo "    Please set I_MPI_ROOT or MPI_ROOT properly."
        echo "    Currently using '$MPI_ARCH_PATH'"
    endif

    _foamAddPath    "$MPI_ARCH_PATH/intel64/bin"
    _foamAddLib     "$MPI_ARCH_PATH/intel64/lib"
    breaksw
endsw


# Add (non-dummy) MPI implementation
# - dummy MPI already in lib-path and has no external libraries
if ( "$FOAM_MPI" != dummy ) then
    if ( $?FOAM_EXT_LIBBIN ) then   # External libraries (allowed to be unset)
        _foamAddLib ${FOAM_EXT_LIBBIN}/${FOAM_MPI}
    endif
    _foamAddLib ${FOAM_LIBBIN}/${FOAM_MPI}
endif

# Alternative to 'mpiBufferSize' controlDict entry
#setenv MPI_BUFFER_SIZE 20000000

#------------------------------------------------------------------------------
